package com.g2room.driving.grap.article.kc;

import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.g2room.driving.model.po.ArticleGrap;
import com.g2room.driving.util.DbUtil;
import com.g2room.driving.util.HtmlUtil;
import com.g2room.driving.util.IpUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;


public class Grap360DataByProxy {
	static Connection connection = null;
	static PreparedStatement  ps = null;
	static Map<String, String> errorUrls = new HashMap<String, String>();
	
	private static final String baseUrl = "http://www.360doc.cn";
	private static final String homePage = "http://www.360doc.cn/taCategoryArticle.aspx?userid=7083970&cid=16&page=";
	private static final String soruce = "车行天下-360文库";
	private static Connection getConn() throws ClassNotFoundException, SQLException {
		if(null == connection) {
			connection = DbUtil.getConn();
		}
		return connection;
	}
	static Map<String, String> errMap = new HashMap<String, String>();
	public static void main(String[] args) throws Exception{
		Grap360DataByProxy gkd = new Grap360DataByProxy();
		gkd.getNav();
		// 最后输出失败的链接到数据库
		insertError2Db();
    }
	
	public static Document getDoc(String url) {
		String ip = IpUtil.getIp();
		Document doc = null;
		try {
			doc =Jsoup.connect(url)
//				  .data("query", "Java")   // 请求参数
					.header("X-Real-IP", ip)
					.header("x-forwarded-for", ip)
					.header("WL-Proxy-Client-IP", ip)
					.userAgent("Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2") // 设置 User-Agent 
//				.cookie("auth", "token") // 设置 cookie 
					.timeout(5000)           // 设置连接超时时间
					.get(); // get方式访问
		} catch (IOException e) {
			e.printStackTrace();
		}
		return doc;
	}
	
	
	/**
	 * 获取头部导航菜单
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getNav() {
		List<String> pages = new ArrayList<String>();
		for(int i=1; i<36; i++) {
			pages.add(homePage + i);
		}
        
        for(String key : pages) {
	        try {
	        	Thread.sleep(IpUtil.genRandomNum() * 500);
	        } catch (InterruptedException e) {
	        	e.printStackTrace();
	        }
	        getPage(key);
        }
	}
	
	/**
	 * 获取分页数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getPage(String url) {
		System.out.println("start get page data!" + url);
		// 获取所有的a标签节点 
		Elements dds = getDoc(url).getElementById("articlelist").children();
		Map<String, String> details = new HashMap<String, String>();
		// 遍历获取下面的子节点
		for(Element el : dds) {
			String title = el.child(0).child(0).text();
			if(DbUtil.titleExist(title, DbUtil.ConnType_KCJQ)) {
				System.out.println(title + "  存在，跳过");
				continue;
			}
			details.put(baseUrl + el.attr("href"), title);
		}
        
        for(String href : details.keySet()) {
        	try {
				Thread.sleep(IpUtil.genRandomNum() * 500);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
        	getDetail(href, details.get(href));
        }
	}
	
	/**
	 * 获取列表数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getDetail(String url, String title) {
		System.out.println("start get list data!url==  " + url);
		try {
			ArticleGrap arti = new  ArticleGrap();
//			arti.setCategoryName(categoryName);
			arti.setTitle(title);
			arti.setSource(soruce);
			arti.setSourceUrl(url);
			Document doc = getDoc(url);
			Element content = doc.getElementById("artcontentdiv");
			String cont = content.html();
			cont = HtmlUtil.delAllTag(cont);
			// 删除a标签
			arti.setContent("<div class=\"article\">" + cont + "</div>");
			
			insert2Db(arti);
		} catch (Exception e) {
			errMap.put(url, title);
			e.printStackTrace();
		}
	}
	
	int sqlCount = 0;
	public void insert2Db(ArticleGrap articles) {
		try {
			if(sqlCount == 0) ps = getConn().prepareStatement("insert into d_article_grap(title,content,type_name,category_name,source,source_url, state) values(?,?,?,?,?,?,?)");
			// title,content,publish_time,type_name,source,source_url
			ps.setString(1, articles.getTitle());
			ps.setString(2, articles.getContent());
			ps.setString(3, articles.getTypeName());
			ps.setString(4, articles.getCategoryName());
			ps.setString(5, soruce);
			ps.setString(6, articles.getSourceUrl());
			ps.setInt(7, 9);
			ps.execute();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	
	public static void insertError2Db() {
		try {
			PreparedStatement  ps = getConn().prepareStatement("insert into d_article_grap_err(title,source,source_url,type_name) values(?,?,?,?)");
			for(String url : errorUrls.keySet()) {
				ps.setString(1, errorUrls.get(url));
				ps.setString(2, soruce);
				ps.setString(3, url);
				ps.setString(4, "导航和列表错误");
				ps.addBatch();
			}
			ps.executeBatch();
			if(null != ps) ps.close();
			if(null != connection) connection.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
}
